package util;


import cn.hutool.core.util.ObjectUtil;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

/**
 *爬虫工具类
 * @author: lilw
 *
* maven 添加依赖：
 *             <dependency>
    *             <groupId>org.apache.httpcomponents</groupId>
    *             <artifactId>httpclient</artifactId>
    *             <version>4.5.13</version>
    *         </dependency>
    *         <dependency>
    *             <groupId>org.jsoup</groupId>
    *             <artifactId>jsoup</artifactId>
    *             <version>1.14.3</version>
    *         </dependency>
 *         <dependency>
 *             <groupId>org.apache.hadoop</groupId>
 *             <artifactId>hadoop-client-runtime</artifactId>
 *             <version>3.3.4</version>
 *         </dependency>
 *         <dependency>
 *             <groupId>cn.hutool</groupId>
 *             <artifactId>hutool-all</artifactId>
 *             <version>5.8.16</version>
 *         </dependency>
 *
 */
public class WebCrawlerUtil {
    private static final int TIMEOUT = 5000; // 超时时间，单位为毫秒


    /**
     * 发送 GET 请求
     * @param url
     * @return
     */
    public static void sendGetRequest(String url, String directory) {
        // 创建 HttpClient 实例
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 创建 HttpGet 请求对象
        HttpGet httpGet = new HttpGet(url);
        // RequestConfig.custom() 创建一个请求配置构建器
        RequestConfig requestConfig = RequestConfig.custom()
        //设置连接超时、请求超时和套接字超时均为常量 TIMEOUT 的值（5000毫秒）
                .setConnectTimeout(TIMEOUT)
                .setConnectionRequestTimeout(TIMEOUT)
                .setSocketTimeout(TIMEOUT)
                //通过 build() 方法构建并返回一个 RequestConfig 对象
                .build();
        //设置 HttpGet 请求的配置信息
        httpGet.setConfig(requestConfig);
        // 发送请求并获取响应
        try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
            //调用 handleResponse() 方法处理响应
            handleResponse(response,directory);
        } catch (IOException e) {
            e.printStackTrace();

        } finally {
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     *  发送 POST 请求
     * @param url
     * @param params
     * @return
     */
    public static void sendPostRequest(String url, List<NameValuePair> params, String directory) {
        // 创建 HttpClient 实例
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 创建 HttpPost 请求对象
        HttpPost httpPost = new HttpPost(url);
        // RequestConfig.custom() 创建一个请求配置构建器
        RequestConfig requestConfig = RequestConfig.custom()
                //设置连接超时、请求超时和套接字超时均为常量 TIMEOUT 的值（5000毫秒）
                .setConnectTimeout(TIMEOUT)
                .setConnectionRequestTimeout(TIMEOUT)
                .setSocketTimeout(TIMEOUT)
                //通过 build() 方法构建并返回一个 RequestConfig 对象
                .build();
        //设置 HttpGet 请求的配置信息
        httpPost.setConfig(requestConfig);
        try {
            if (params!= null) {
                httpPost.setEntity(new UrlEncodedFormEntity(params, "UTF-8"));
            }
            try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
                 handleResponse(response,directory);
            }
        } catch (IOException e) {
            e.printStackTrace();

        } finally {
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 处理响应
     * @param response
     * @param directory  保存文件的目录
     * @return
     * @throws IOException
     *
     */
    private static Map<String, Object> handleResponse(CloseableHttpResponse response, String directory) throws IOException {
        Map<String, Object> map = new HashMap<>();
        List<String> list = new ArrayList<>();
        int statusCode = response.getStatusLine().getStatusCode();
        if (statusCode == 200) {
            HttpEntity entity = response.getEntity();
            if (entity!= null) {
                // 将实体内容转换为字符串
                String html = EntityUtils.toString(entity, "UTF-8");
                // 使用 Jsoup 解析 HTML
                Document doc = Jsoup.parse(html);
                // 查找 HTML 中的元素，这里以查找所有的 <a> 标签为例
                Elements links = doc.select("a");
                // 查找 HTML 中的 <img> 标签
                Elements imgElements = doc.select("img");
                // 创建保存信息的目录
                createDirectoryIfNotExists(directory);
                // 指定保存图片的目录
                String imageDirectory = directory+"images/";
                // 创建保存图片的目录
                createDirectoryIfNotExists(imageDirectory);

                for (Element link : links) {
                    // 输出链接的 href 属性
                    if(ObjectUtil.isNotEmpty(link.text())){
                        map.put(link.text(),link.attr("href"));
                    }else {
                        list.add(link.attr("href"));
                    }
                }
                for (Element img : imgElements) {
                    String imgUrl = img.attr("src");
                    if (ObjectUtil.isNotEmpty(imgUrl)) {
                        downloadImage(imgUrl,imageDirectory );
                    }
                }

                writeMapToFile(map, directory + UUID.randomUUID()+"output_map.txt");
                writeListToFile(list, directory + UUID.randomUUID()+"output_list.txt");
                return map;
            }
        } else {
            System.out.println("Failed to fetch page. Status code: " + statusCode);
        }
        return null;
    }



    /**
     * 创建目录的方法
     * @param directory
     * @throws IOException
     */
    private static void createDirectoryIfNotExists(String directory) throws IOException {
        Path path = Paths.get(directory);
        if (!Files.exists(path)) {
            Files.createDirectories(path);
        }
    }



    /**
     * 下载图片的方法
     * @param imgUrl
     * @param imageDirectory
     */
    private static void downloadImage(String imgUrl, String imageDirectory) {
        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
            // 创建 URL 对象
            URL url = new URL(imgUrl);
            // 生成图片文件名
            String fileName = imageDirectory + UUID.randomUUID() + ".jpg";
            try (InputStream in = url.openStream();
                 OutputStream out = new FileOutputStream(fileName)) {
                byte[] buffer = new byte[4096];
                int bytesRead;
                while ((bytesRead = in.read(buffer)) != -1) {
                    out.write(buffer, 0, bytesRead);
                }
                System.out.println("Downloaded: " + imgUrl);
            }
        } catch (IOException e) {
            System.err.println("下载图片错误: " + imgUrl + " - " + e.getMessage());
        }
    }


    /**
     * 将 Map 写入文件
     * @param map
     * @param filePath
     */
    private static void writeMapToFile(Map<String, Object> map, String filePath) {
        File file = new File(filePath);
        // 确保父目录存在，如果不存在则创建
        File parentDir = file.getParentFile();
        if (parentDir!= null &&!parentDir.exists()) {
            parentDir.mkdirs();
        }
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
            for (Map.Entry<String, Object> entry : map.entrySet()) {
                writer.write(entry.getKey() + ":  \n" + entry.getValue());
                writer.newLine();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


    /**
     * 将 List 写入文件
     * @param list
     * @param filePath
     */
    private static void writeListToFile(List<String> list, String filePath) {
        File file = new File(filePath);
        // 确保父目录存在，如果不存在则创建
        File parentDir = file.getParentFile();
        if (parentDir!= null &&!parentDir.exists()) {
            parentDir.mkdirs();
        }
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
            for (String item : list) {
                writer.write(item);
                writer.newLine();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }





}